{
  "cells": [
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "\"\"\"\nin_args:\n:param test_size:FLOAT:0.2 测试数据集大小\n:param random_state:INTEGER:42 随机数\n\nout_args:\n\n:param test_mse:training_result['performance_metrics'][\"test_mse\"] 测试集mse\n\n:param test_r2:training_result['performance_metrics'][\"test_r2\"] 测试集r2\n\ndataset:\n:BCDP_702217_FJSJJ_sql:/task/dataset.csv  \n\noutput:\n:/task/saved_model/\n\"\"\"\nds_ename = \"BCDP_702217_FJSJJ_sql\"\nfrom bexk_da_sdk import bexk_da_repo\nbexk_da_repo.load(\"DS:PD\",ds_ename,\"./dataset.csv\")",
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": "2025-11-21 22:20:23,823 - INFO - 本地仓库文件无需更新，已使用对应版本\n",
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "#输入参数\nimport os\nimport json\ntask_args=json.loads(os.getenv(\"BEDE_GLOBAL_TASK_ARGS\")) if os.getenv(\"BEDE_GLOBAL_TASK_ARGS\") is not None else {}\nparam_test_size=float(task_args.get(\"test_size\",0.2))\nparam_random_state=int(task_args.get(\"random_state\",42))\n",
      "execution_count": 26,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "scrolled": false
      },
      "cell_type": "code",
      "source": "import os\nimport logging\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split, GridSearchCV\nfrom sklearn.linear_model import LinearRegression, Ridge, Lasso\nfrom sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.preprocessing import StandardScaler, PolynomialFeatures\nfrom sklearn.feature_selection import SelectKBest, f_regression\nfrom sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error\nimport joblib\n# import warnings\n# warnings.filterwarnings('ignore')\nos.environ['LOKY_MAX_CPU_COUNT'] = '1'  # 限制loky后端最多使用1个CPU核心\n\n# 配置日志\nlogging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\nlogger = logging.getLogger(__name__)\n\ndef load_data(data_path):\n    \"\"\"加载数据并进行初步验证\"\"\"\n    try:\n        if not os.path.exists(data_path):\n            raise FileNotFoundError(f\"数据文件不存在: {data_path}\")\n        \n        data = pd.read_csv(data_path)\n        logger.info(f\"成功加载数据，数据形状: {data.shape}\")\n        logger.info(f\"数据列名: {list(data.columns)}\")\n        logger.info(f\"数据基本信息:\\n{data.info()}\")\n        logger.info(f\"数据描述性统计:\\n{data.describe().round(2)}\")\n        \n        return data\n    except Exception as e:\n        logger.error(f\"加载数据失败: {str(e)}\", exc_info=True)\n        raise\n\ndef preprocess_data(data, feature_columns, target_column):\n    \"\"\"数据预处理：空值填充、异常值剔除\"\"\"\n    # 复制数据避免修改原始数据\n    data_processed = data[feature_columns + target_column].copy()\n    \n    logger.info(\"开始数据预处理...\")\n    \n    # 1. 空值填充\n    logger.info(f\"空值统计:\\n{data_processed.isnull().sum()}\")\n    for col in data_processed.columns:\n        null_count = data_processed[col].isnull().sum()\n        if null_count > 0:\n            # 数值型特征用中位数填充（更鲁棒，不受异常值影响）\n            median_val = data_processed[col].median()\n            data_processed[col].fillna(median_val, inplace=True)\n            logger.info(f\"填充空值 - 列 {col}: 空值数量 {null_count}, 使用中位数 {median_val:.2f}\")\n    \n    # 2. 异常值剔除（IQR方法）\n    logger.info(\"开始剔除异常值...\")\n    before_outlier_removal = len(data_processed)\n    \n    # 对每个特征列进行异常值处理\n    for col in feature_columns:\n        # 计算IQR\n        Q1 = data_processed[col].quantile(0.25)\n        Q3 = data_processed[col].quantile(0.75)\n        IQR = Q3 - Q1\n        lower_bound = Q1 - 1.5 * IQR\n        upper_bound = Q3 + 1.5 * IQR\n        \n        # 保留在正常范围内的数据\n        data_processed = data_processed[\n            (data_processed[col] >= lower_bound) & (data_processed[col] <= upper_bound)\n        ]\n    \n    after_outlier_removal = len(data_processed)\n    outlier_count = before_outlier_removal - after_outlier_removal\n    logger.info(f\"异常值剔除完成 - 原始数据量: {before_outlier_removal}, 处理后数据量: {after_outlier_removal}, \"\n                f\"剔除异常值数量: {outlier_count}, 剔除比例: {outlier_count/before_outlier_removal:.2%}\")\n    \n    # 3. 检查目标变量合理性（房屋价值不应为负）\n    target_col = target_column[0]\n    invalid_target = (data_processed[target_col] <= 0).sum()\n    if invalid_target > 0:\n        data_processed = data_processed[data_processed[target_col] > 0]\n        logger.info(f\"剔除无效目标值（<=0）: {invalid_target} 个\")\n    \n    logger.info(f\"最终处理后数据量: {len(data_processed)}\")\n    \n    return data_processed\n\ndef feature_engineering(X, y):\n    \"\"\"特征工程：特征标准化、特征交互、特征选择\"\"\"\n    logger.info(\"开始特征工程...\")\n    \n    # 1. 特征标准化（对线性模型很重要）\n    scaler = StandardScaler()\n    X_scaled = scaler.fit_transform(X)\n    X_scaled_df = pd.DataFrame(X_scaled, columns=X.columns, index=X.index)\n    \n    # 2. 特征交互（增加有意义的交互特征，使用独特的命名避免冲突）\n    interaction_features_list = ['MEDINC', 'AVE_ROOMS', 'POPULATION']\n    interaction_features_list = [col for col in interaction_features_list if col in X.columns]\n    n_interaction_feats = len(interaction_features_list)\n    \n    if n_interaction_feats >= 2:\n        # 生成多项式特征（仅交互项）\n        poly = PolynomialFeatures(degree=2, include_bias=False, interaction_only=True)\n        interaction_features = poly.fit_transform(X_scaled_df[interaction_features_list])\n        \n        # 生成交互特征名称（完全不依赖内部属性）\n        interaction_feature_names = []\n        # 遍历所有可能的特征对（i < j 避免重复）\n        for i in range(n_interaction_feats):\n            for j in range(i + 1, n_interaction_feats):\n                feat1 = interaction_features_list[i]\n                feat2 = interaction_features_list[j]\n                interaction_feature_names.append(f\"inter_{feat1}_{feat2}\")\n        \n        # 验证生成的特征数量是否匹配\n        if len(interaction_feature_names) != interaction_features.shape[1]:\n            logger.warning(f\"交互特征名称数量与实际特征数量不匹配，使用默认命名\")\n            interaction_feature_names = [f\"inter_{i}\" for i in range(interaction_features.shape[1])]\n        \n        interaction_df = pd.DataFrame(interaction_features, columns=interaction_feature_names, index=X.index)\n        \n        # 组合原始特征和交互特征\n        X_engineered = pd.concat([X_scaled_df, interaction_df], axis=1)\n        logger.info(f\"特征工程后特征数量: {X_engineered.shape[1]}, 原始特征数量: {X.shape[1]}, 交互特征数量: {len(interaction_feature_names)}\")\n    else:\n        # 如果可用于交互的特征少于2个，不生成交互特征\n        logger.warning(f\"可用于交互的特征数量不足（{n_interaction_feats} 个），跳过交互特征生成\")\n        X_engineered = X_scaled_df\n        interaction_feature_names = []\n        poly = None\n    \n    # 3. 特征选择（选择最重要的特征，最多10个）\n    k = min(10, X_engineered.shape[1])\n    selector = SelectKBest(score_func=f_regression, k=k)\n    X_selected = selector.fit_transform(X_engineered, y.values.ravel())\n    \n    # 获取选择的特征名称\n    selected_features = X_engineered.columns[selector.get_support()]\n    logger.info(f\"特征选择后保留的特征 ({len(selected_features)} 个): {list(selected_features)}\")\n    \n    # 验证没有重复特征\n    if len(selected_features) != len(set(selected_features)):\n        logger.warning(\"发现重复特征，自动去重\")\n        selected_features = list(dict.fromkeys(selected_features))  # 保持顺序去重\n        # 重新选择去重后的特征\n        X_selected = X_engineered[selected_features].values\n        logger.info(f\"去重后保留的特征 ({len(selected_features)} 个): {list(selected_features)}\")\n    \n    X_selected_df = pd.DataFrame(X_selected, columns=selected_features, index=X_engineered.index)\n    \n    # 保存特征工程相关对象和信息\n    feature_engineering_objects = {\n        'scaler': scaler,\n        'poly': poly,  # 可能为None\n        'selector': selector,\n        'selected_features': selected_features,\n        'interaction_features_list': interaction_features_list,\n        'interaction_feature_names': interaction_feature_names,\n        'original_feature_columns': X.columns.tolist()\n    }\n    \n    return X_selected_df, feature_engineering_objects\n\ndef build_models():\n    \"\"\"构建多种回归模型及超参数网格\"\"\"\n    models = {\n        'LinearRegression': {\n            'model': LinearRegression(),\n            'params': {}\n        },\n        'Ridge': {\n            'model': Ridge(random_state=42),\n            'params': {\n                'alpha': [0.1, 1.0, 5.0, 10.0, 20.0]\n            }\n        },\n        'Lasso': {\n            'model': Lasso(random_state=42, max_iter=10000),\n            'params': {\n                'alpha': [0.01, 0.1, 1.0, 5.0, 10.0]\n            }\n        },\n        'DecisionTree': {\n            'model': DecisionTreeRegressor(random_state=42),\n            'params': {\n                'max_depth': [3, 5, 7, 10, None],\n                'min_samples_split': [2, 5, 10],\n                'min_samples_leaf': [1, 2, 5]\n            }\n        },\n\n        'RandomForest': {\n            'model': RandomForestRegressor(random_state=42, n_jobs=1,min_samples_split= 2,  min_samples_leaf= 1),\n            'params': {\n                'n_estimators': [200]\n            }\n        },\n        'GradientBoosting': {\n            'model': GradientBoostingRegressor(random_state=42,learning_rate = 0.1,subsample= 1.0,  criterion= \"friedman_mse\",  min_samples_split= 2,  min_samples_leaf= 1,  alpha= 0.9),\n            'params': {\n                'n_estimators': [100, 200],\n                'max_depth': [3, 5]\n            }\n        }\n    }\n    return models\n\ndef train_and_evaluate_models(X_train, X_test, y_train, y_test, models):\n    \"\"\"训练多个模型并评估性能\"\"\"\n    logger.info(\"开始训练和评估多个模型...\")\n    results = []\n    \n    for model_name, model_info in models.items():\n        logger.info(f\"\\n{'='*50}\")\n        logger.info(f\"训练模型: {model_name}\")\n        logger.info(f\"{'='*50}\")\n        logger.info(f\"使用特征数量: {X_train.shape[1]}\")\n        \n        # 使用网格搜索进行超参数调优\n        grid_search = GridSearchCV(\n            estimator=model_info['model'],\n            param_grid=model_info['params'],\n            cv=5,\n            scoring='r2',\n            n_jobs=-1,\n            verbose=0  # 关闭详细输出，避免日志混乱\n        )\n        \n        # 训练模型\n        grid_search.fit(X_train, y_train.values.ravel())\n        \n        # 最佳模型\n        best_model = grid_search.best_estimator_\n        logger.info(f\"最佳超参数: {grid_search.best_params_}\")\n        \n        # 预测\n        y_train_pred = best_model.predict(X_train)\n        y_test_pred = best_model.predict(X_test)\n        \n        # 评估指标\n        metrics = {\n            'model_name': model_name,\n            'best_params': grid_search.best_params_,\n            'train_mse': mean_squared_error(y_train, y_train_pred),\n            'test_mse': mean_squared_error(y_test, y_test_pred),\n            'train_rmse': np.sqrt(mean_squared_error(y_train, y_train_pred)),\n            'test_rmse': np.sqrt(mean_squared_error(y_test, y_test_pred)),\n            'train_mae': mean_absolute_error(y_train, y_train_pred),\n            'test_mae': mean_absolute_error(y_test, y_test_pred),\n            'train_r2': r2_score(y_train, y_train_pred),\n            'test_r2': r2_score(y_test, y_test_pred),\n            'cv_best_score': grid_search.best_score_,\n            'model': best_model,\n            'feature_count': X_train.shape[1]\n        }\n        \n        # 打印结果\n        logger.info(f\"训练集 - MSE: {metrics['train_mse']:.4f}, RMSE: {metrics['train_rmse']:.4f}, \"\n                    f\"MAE: {metrics['train_mae']:.4f}, R²: {metrics['train_r2']:.4f}\")\n        logger.info(f\"测试集 - MSE: {metrics['test_mse']:.4f}, RMSE: {metrics['test_rmse']:.4f}, \"\n                    f\"MAE: {metrics['test_mae']:.4f}, R²: {metrics['test_r2']:.4f}\")\n        logger.info(f\"交叉验证最佳得分 (R²): {metrics['cv_best_score']:.4f}\")\n        \n        results.append(metrics)\n    \n    # 转换结果为DataFrame便于查看\n    results_df = pd.DataFrame(results)\n    results_df = results_df.sort_values('test_r2', ascending=False).reset_index(drop=True)\n    \n    # 打印所有模型性能对比\n    logger.info(\"\\n\" + \"=\"*80)\n    logger.info(\"所有模型性能对比（按测试集R²排序）:\")\n    logger.info(\"=\"*80)\n    comparison_df = results_df[['model_name', 'test_r2', 'test_rmse', 'cv_best_score', 'feature_count']].copy()\n    comparison_df['test_r2'] = comparison_df['test_r2'].round(4)\n    comparison_df['test_rmse'] = comparison_df['test_rmse'].round(4)\n    comparison_df['cv_best_score'] = comparison_df['cv_best_score'].round(4)\n    logger.info(comparison_df.to_string(index=False))\n    \n    return results, results_df\n\ndef select_best_model(results_df):\n    \"\"\"选择最优模型（优先考虑测试集R²，兼顾泛化能力）\"\"\"\n    # 选择测试集R²最高的模型\n    best_idx = results_df['test_r2'].idxmax()\n    best_model_info = results_df.iloc[best_idx]\n    \n    logger.info(\"\\n\" + \"=\"*80)\n    logger.info(f\"最优模型: {best_model_info['model_name']}\")\n    logger.info(\"=\"*80)\n    logger.info(f\"最优模型超参数: {best_model_info['best_params']}\")\n    logger.info(f\"测试集 R²: {best_model_info['test_r2']:.4f}\")\n    logger.info(f\"测试集 RMSE: {best_model_info['test_rmse']:.4f}\")\n    logger.info(f\"交叉验证最佳得分: {best_model_info['cv_best_score']:.4f}\")\n    logger.info(f\"训练集 R²: {best_model_info['train_r2']:.4f}\")\n    logger.info(f\"使用特征数量: {best_model_info['feature_count']}\")\n    \n    return best_model_info['model'], best_model_info['model_name']\n\ndef save_model_and_artifacts(best_model, feature_engineering_objects, results_df, model_dir):\n    \"\"\"保存最优模型和相关组件\"\"\"\n    os.makedirs(model_dir, exist_ok=True)\n    \n    # 保存最佳模型\n    model_filename = f\"best_model_{best_model.__class__.__name__}.pkl\"\n    model_path = os.path.join(model_dir, model_filename)\n    joblib.dump(best_model, model_path)\n    \n    # 保存特征工程组件\n    fe_path = os.path.join(model_dir, \"feature_engineering_objects.pkl\")\n    joblib.dump(feature_engineering_objects, fe_path)\n    \n    # 保存模型性能报告\n    report_path = os.path.join(model_dir, \"model_performance_report.csv\")\n    # 处理字典类型的列，便于保存为CSV\n    results_save = results_df.copy()\n    results_save['best_params'] = results_save['best_params'].astype(str)\n    results_save.drop('model', axis=1, inplace=True)  # 移除模型对象列\n    results_save.to_csv(report_path, index=False, encoding='utf-8')\n    \n    logger.info(f\"\\n模型保存完成:\")\n    logger.info(f\"最佳模型路径: {model_path}\")\n    logger.info(f\"特征工程组件路径: {fe_path}\")\n    logger.info(f\"性能报告路径: {report_path}\")\n    \n    return model_path\n\ndef train_pipeline(data_path, test_size=0.2, random_state=42, model_dir=\"./saved_model\"):\n    \"\"\"完整训练流程流水线\"\"\"\n    try:\n        logger.info(\"=\"*80)\n        logger.info(\"开始模型训练流水线\")\n        logger.info(\"=\"*80)\n        \n        # 1. 加载数据\n        data = load_data(data_path)\n        \n        # 2. 定义特征列和目标列\n        feature_columns = [\"MEDINC\", \"HOUSE_AGE\", \"AVE_ROOMS\", \"AVE_BEDRMS\", \n                          \"AVE_OCCUP\", \"LATITUDE\", \"LONGITUDE\", \"POPULATION\"]\n        target_column = [\"MED_HOUSE_VAL\"]\n        \n        # 检查必要的列是否存在\n        missing_features = [col for col in feature_columns if col not in data.columns]\n        missing_target = [col for col in target_column if col not in data.columns]\n        \n        if missing_features:\n            raise ValueError(f\"数据中缺少必要的特征列: {missing_features}\")\n        if missing_target:\n            raise ValueError(f\"数据中缺少目标列: {missing_target}\")\n        \n        # 3. 数据预处理\n        data_processed = preprocess_data(data, feature_columns, target_column)\n        \n        # 4. 分离特征和目标变量\n        X = data_processed[feature_columns]\n        y = data_processed[target_column]\n        logger.info(f\"特征矩阵形状: {X.shape}, 目标变量形状: {y.shape}\")\n        \n        # 5. 特征工程（正确传入y参数）\n        X_engineered, fe_objects = feature_engineering(X, y)\n        logger.info(f\"特征工程后最终特征矩阵形状: {X_engineered.shape}\")\n        \n        # 6. 划分训练集和测试集\n        X_train, X_test, y_train, y_test = train_test_split(\n            X_engineered, y, test_size=test_size, random_state=random_state, shuffle=True\n        )\n        logger.info(f\"训练集形状: {X_train.shape}, 测试集形状: {X_test.shape}\")\n        \n        # 7. 构建模型集合\n        models = build_models()\n        \n        # 8. 训练和评估所有模型\n        results, results_df = train_and_evaluate_models(X_train, X_test, y_train, y_test, models)\n        \n        # 9. 选择最优模型\n        best_model, best_model_name = select_best_model(results_df)\n        \n        # 10. 保存模型和相关组件\n        model_path = save_model_and_artifacts(best_model, fe_objects, results_df, model_dir)\n        \n        logger.info(\"\\n\" + \"=\"*80)\n        logger.info(\"模型训练流水线完成！\")\n        logger.info(\"=\"*80)\n        \n        # 返回最优模型信息\n        best_metrics = results_df.iloc[results_df['test_r2'].idxmax()].to_dict()\n        return {\n            'best_model_path': model_path,\n            'best_model_name': best_model_name,\n            'performance_metrics': {\n                'test_r2': best_metrics['test_r2'],\n                'test_rmse': best_metrics['test_rmse'],\n                'test_mse': best_metrics['test_mse'],\n                'test_mae': best_metrics['test_mae'],\n                'cv_best_score': best_metrics['cv_best_score']\n            },\n            'feature_engineering_info': {\n                'selected_features': fe_objects['selected_features'],\n                'feature_count': len(fe_objects['selected_features']),\n                'interaction_features': fe_objects['interaction_features_list']\n            }\n        }\n        \n    except Exception as e:\n        logger.error(f\"训练流水线失败: {str(e)}\", exc_info=True)\n        raise\n\ndef predict_with_best_model(model_dir, X_new):\n    \"\"\"使用训练好的最优模型进行预测\"\"\"\n    # 加载模型和特征工程组件\n    try:\n        # 找到最佳模型文件\n        model_files = [f for f in os.listdir(model_dir) if f.startswith(\"best_model_\") and f.endswith(\".pkl\")]\n        if not model_files:\n            raise FileNotFoundError(\"未找到训练好的最佳模型\")\n        \n        model_path = os.path.join(model_dir, model_files[0])\n        fe_path = os.path.join(model_dir, \"feature_engineering_objects.pkl\")\n        \n        best_model = joblib.load(model_path)\n        fe_objects = joblib.load(fe_path)\n        \n        logger.info(f\"成功加载模型: {model_path}\")\n        logger.info(f\"模型期望特征数量: {best_model.n_features_in_}\")\n        logger.info(f\"训练时选择的特征: {fe_objects['selected_features']}\")\n        logger.info(f\"特征数量: {len(fe_objects['selected_features'])}\")\n        \n        # 验证输入数据的特征列是否完整\n        required_features = fe_objects['original_feature_columns']\n        missing_features = [col for col in required_features if col not in X_new.columns]\n        if missing_features:\n            raise ValueError(f\"输入数据缺少必要的原始特征: {missing_features}\")\n        \n        # 特征处理（与训练时保持完全一致）\n        # 1. 标准化\n        X_scaled = fe_objects['scaler'].transform(X_new[required_features])\n        X_scaled_df = pd.DataFrame(X_scaled, columns=required_features, index=X_new.index)\n        \n        # 2. 生成交互特征（如果训练时生成了的话）\n        X_engineered = X_scaled_df.copy()\n        if fe_objects['poly'] is not None and len(fe_objects['interaction_feature_names']) > 0:\n            interaction_features = fe_objects['poly'].transform(X_scaled_df[fe_objects['interaction_features_list']])\n            interaction_feature_names = fe_objects['interaction_feature_names']\n            interaction_df = pd.DataFrame(interaction_features, columns=interaction_feature_names, index=X_new.index)\n            \n            # 组合特征\n            X_engineered = pd.concat([X_engineered, interaction_df], axis=1)\n            logger.info(f\"生成交互特征数量: {len(interaction_feature_names)}\")\n        \n        logger.info(f\"特征工程后总特征数量: {X_engineered.shape[1]}\")\n        \n        # 3. 严格按照训练时选择的特征列表筛选\n        selected_features = fe_objects['selected_features']\n        missing_selected = [col for col in selected_features if col not in X_engineered.columns]\n        \n        if missing_selected:\n            raise ValueError(f\"生成的特征中缺少训练时选择的特征: {missing_selected}\")\n        \n        X_selected = X_engineered[selected_features]\n        logger.info(f\"筛选后特征数量: {X_selected.shape[1]}\")\n        \n        # 验证特征数量是否匹配\n        if X_selected.shape[1] != best_model.n_features_in_:\n            raise ValueError(f\"特征数量不匹配 - 提供: {X_selected.shape[1]}, 期望: {best_model.n_features_in_}\")\n        \n        # 预测\n        y_pred = best_model.predict(X_selected)\n        logger.info(f\"预测完成，生成 {len(y_pred)} 个预测结果\")\n        \n        return y_pred\n    \n    except Exception as e:\n        logger.error(f\"预测失败: {str(e)}\", exc_info=True)\n        raise\n",
      "execution_count": 27,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "\n# 运行训练流水线\ntry:\n    # 强制删除旧的模型目录，确保使用全新的训练逻辑\n    if os.path.exists(\"./saved_model\"):\n        import shutil\n        shutil.rmtree(\"./saved_model\")\n        logger.info(\"已删除旧的模型目录，将重新训练模型\")\n\n    training_result = train_pipeline(\n        data_path=\"./dataset.csv\",  # 数据文件路径\n        test_size=0.2,             # 测试集比例\n        random_state=42,           # 随机种子\n        model_dir=\"./saved_model\"  # 模型保存目录\n    )\n\n    # 打印最终结果摘要\n    print(\"\\n\" + \"=\"*80)\n    print(\"训练结果摘要\")\n    print(\"=\"*80)\n    print(f\"最优模型: {training_result['best_model_name']}\")\n    print(f\"最优模型路径: {training_result['best_model_path']}\")\n    print(f\"\\n性能指标:\")\n    print(f\"  测试集 R² 得分: {training_result['performance_metrics']['test_r2']:.4f}\")\n    print(f\"  测试集 RMSE: {training_result['performance_metrics']['test_rmse']:.4f}\")\n    print(f\"  测试集 MSE: {training_result['performance_metrics']['test_mse']:.4f}\")\n    print(f\"  测试集 MAE: {training_result['performance_metrics']['test_mae']:.4f}\")\n    print(f\"  交叉验证最佳得分: {training_result['performance_metrics']['cv_best_score']:.4f}\")\n    print(f\"\\n特征工程信息:\")\n    print(f\"  选择的特征数量: {training_result['feature_engineering_info']['feature_count']}\")\n    print(f\"  选择的特征: {training_result['feature_engineering_info']['selected_features']}\")\n    test_mse=training_result['performance_metrics']['test_mse']\n    test_r2=training_result['performance_metrics']['test_r2']\n\n\nexcept Exception as e:\n    print(f\"程序执行失败: {str(e)}\")",
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": "2025-11-21 22:20:23,902 - INFO - 已删除旧的模型目录，将重新训练模型\n2025-11-21 22:20:23,903 - INFO - ================================================================================\n2025-11-21 22:20:23,904 - INFO - 开始模型训练流水线\n2025-11-21 22:20:23,904 - INFO - ================================================================================\n2025-11-21 22:20:23,967 - INFO - 成功加载数据，数据形状: (20639, 10)\n2025-11-21 22:20:23,968 - INFO - 数据列名: ['DATA_SN', 'MEDINC', 'HOUSE_AGE', 'AVE_ROOMS', 'AVE_BEDRMS', 'AVE_OCCUP', 'LATITUDE', 'LONGITUDE', 'MED_HOUSE_VAL', 'POPULATION']\n2025-11-21 22:20:23,981 - INFO - 数据基本信息:\nNone\n2025-11-21 22:20:24,013 - INFO - 数据描述性统计:\n        DATA_SN    MEDINC  HOUSE_AGE  AVE_ROOMS  AVE_BEDRMS  AVE_OCCUP  \\\ncount  20639.00  20639.00   20639.00   20639.00    20639.00   20639.00   \nmean   10320.00      3.87      28.64       5.43        1.10       3.07   \nstd     5958.11      1.90      12.59       2.47        0.47      10.39   \nmin        1.00      0.50       1.00       0.85        0.33       0.69   \n25%     5160.50      2.56      18.00       4.44        1.01       2.43   \n50%    10320.00      3.53      29.00       5.23        1.05       2.82   \n75%    15479.50      4.74      37.00       6.05        1.10       3.28   \nmax    20639.00     15.00      52.00     141.91       34.07    1243.33   \n\n       LATITUDE  LONGITUDE  MED_HOUSE_VAL  POPULATION  \ncount  20639.00   20639.00       20639.00    20639.00  \nmean      35.63    -119.57           2.07     1425.48  \nstd        2.14       2.00           1.15     1132.49  \nmin       32.54    -124.35           0.15        3.00  \n25%       33.93    -121.80           1.20      787.00  \n50%       34.26    -118.49           1.80     1166.00  \n75%       37.71    -118.01           2.65     1725.00  \nmax       41.95    -114.31           5.00    35682.00  \n2025-11-21 22:20:24,017 - INFO - 开始数据预处理...\n2025-11-21 22:20:24,019 - INFO - 空值统计:\nMEDINC           0\nHOUSE_AGE        0\nAVE_ROOMS        0\nAVE_BEDRMS       0\nAVE_OCCUP        0\nLATITUDE         0\nLONGITUDE        0\nPOPULATION       0\nMED_HOUSE_VAL    0\ndtype: int64\n2025-11-21 22:20:24,024 - INFO - 开始剔除异常值...\n2025-11-21 22:20:24,078 - INFO - 异常值剔除完成 - 原始数据量: 20639, 处理后数据量: 16799, 剔除异常值数量: 3840, 剔除比例: 18.61%\n2025-11-21 22:20:24,079 - INFO - 最终处理后数据量: 16799\n2025-11-21 22:20:24,081 - INFO - 特征矩阵形状: (16799, 8), 目标变量形状: (16799, 1)\n2025-11-21 22:20:24,082 - INFO - 开始特征工程...\n2025-11-21 22:20:24,090 - WARNING - 交互特征名称数量与实际特征数量不匹配，使用默认命名\n2025-11-21 22:20:24,092 - INFO - 特征工程后特征数量: 14, 原始特征数量: 8, 交互特征数量: 6\n",
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": "<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 20639 entries, 0 to 20638\nData columns (total 10 columns):\n #   Column         Non-Null Count  Dtype  \n---  ------         --------------  -----  \n 0   DATA_SN        20639 non-null  int64  \n 1   MEDINC         20639 non-null  float64\n 2   HOUSE_AGE      20639 non-null  float64\n 3   AVE_ROOMS      20639 non-null  float64\n 4   AVE_BEDRMS     20639 non-null  float64\n 5   AVE_OCCUP      20639 non-null  float64\n 6   LATITUDE       20639 non-null  float64\n 7   LONGITUDE      20639 non-null  float64\n 8   MED_HOUSE_VAL  20639 non-null  float64\n 9   POPULATION     20639 non-null  float64\ndtypes: float64(9), int64(1)\nmemory usage: 1.6 MB\n",
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": "2025-11-21 22:20:24,361 - INFO - 特征选择后保留的特征 (10 个): ['MEDINC', 'HOUSE_AGE', 'AVE_ROOMS', 'AVE_BEDRMS', 'AVE_OCCUP', 'LATITUDE', 'inter_0', 'inter_1', 'inter_3', 'inter_4']\n2025-11-21 22:20:24,363 - INFO - 特征工程后最终特征矩阵形状: (16799, 10)\n2025-11-21 22:20:24,371 - INFO - 训练集形状: (13439, 10), 测试集形状: (3360, 10)\n2025-11-21 22:20:24,372 - INFO - 开始训练和评估多个模型...\n2025-11-21 22:20:24,373 - INFO - \n==================================================\n2025-11-21 22:20:24,374 - INFO - 训练模型: LinearRegression\n2025-11-21 22:20:24,374 - INFO - ==================================================\n2025-11-21 22:20:24,375 - INFO - 使用特征数量: 10\n2025-11-21 22:20:34,661 - INFO - 最佳超参数: {}\n2025-11-21 22:20:35,164 - INFO - 训练集 - MSE: 0.4589, RMSE: 0.6774, MAE: 0.5057, R²: 0.5990\n2025-11-21 22:20:35,165 - INFO - 测试集 - MSE: 0.4742, RMSE: 0.6886, MAE: 0.5101, R²: 0.5869\n2025-11-21 22:20:35,165 - INFO - 交叉验证最佳得分 (R²): 0.5978\n2025-11-21 22:20:35,166 - INFO - \n==================================================\n2025-11-21 22:20:35,166 - INFO - 训练模型: Ridge\n2025-11-21 22:20:35,167 - INFO - ==================================================\n2025-11-21 22:20:35,167 - INFO - 使用特征数量: 10\n2025-11-21 22:20:44,161 - INFO - 最佳超参数: {'alpha': 20.0}\n2025-11-21 22:20:44,565 - INFO - 训练集 - MSE: 0.4589, RMSE: 0.6774, MAE: 0.5057, R²: 0.5990\n2025-11-21 22:20:44,566 - INFO - 测试集 - MSE: 0.4742, RMSE: 0.6886, MAE: 0.5101, R²: 0.5869\n2025-11-21 22:20:44,567 - INFO - 交叉验证最佳得分 (R²): 0.5978\n2025-11-21 22:20:44,567 - INFO - \n==================================================\n2025-11-21 22:20:44,568 - INFO - 训练模型: Lasso\n2025-11-21 22:20:44,568 - INFO - ==================================================\n2025-11-21 22:20:44,569 - INFO - 使用特征数量: 10\n2025-11-21 22:20:55,362 - INFO - 最佳超参数: {'alpha': 0.01}\n2025-11-21 22:20:55,866 - INFO - 训练集 - MSE: 0.4601, RMSE: 0.6783, MAE: 0.5062, R²: 0.5980\n2025-11-21 22:20:55,867 - INFO - 测试集 - MSE: 0.4752, RMSE: 0.6893, MAE: 0.5106, R²: 0.5860\n2025-11-21 22:20:55,961 - INFO - 交叉验证最佳得分 (R²): 0.5969\n2025-11-21 22:20:55,962 - INFO - \n==================================================\n2025-11-21 22:20:55,962 - INFO - 训练模型: DecisionTree\n2025-11-21 22:20:55,963 - INFO - ==================================================\n2025-11-21 22:20:55,964 - INFO - 使用特征数量: 10\n2025-11-21 22:21:13,050 - INFO - 最佳超参数: {'max_depth': 7, 'min_samples_leaf': 5, 'min_samples_split': 2}\n2025-11-21 22:21:13,070 - INFO - 训练集 - MSE: 0.3795, RMSE: 0.6161, MAE: 0.4479, R²: 0.6683\n2025-11-21 22:21:13,071 - INFO - 测试集 - MSE: 0.4486, RMSE: 0.6698, MAE: 0.4804, R²: 0.6091\n2025-11-21 22:21:13,071 - INFO - 交叉验证最佳得分 (R²): 0.6114\n2025-11-21 22:21:13,072 - INFO - \n==================================================\n2025-11-21 22:21:13,072 - INFO - 训练模型: RandomForest\n2025-11-21 22:21:13,073 - INFO - ==================================================\n2025-11-21 22:21:13,073 - INFO - 使用特征数量: 10\n2025-11-21 22:23:45,963 - INFO - 最佳超参数: {'n_estimators': 200}\n2025-11-21 22:23:46,877 - INFO - 训练集 - MSE: 0.0459, RMSE: 0.2143, MAE: 0.1504, R²: 0.9599\n2025-11-21 22:23:46,879 - INFO - 测试集 - MSE: 0.3454, RMSE: 0.5877, MAE: 0.4125, R²: 0.6991\n2025-11-21 22:23:46,879 - INFO - 交叉验证最佳得分 (R²): 0.7005\n2025-11-21 22:23:46,880 - INFO - \n==================================================\n2025-11-21 22:23:46,880 - INFO - 训练模型: GradientBoosting\n2025-11-21 22:23:46,881 - INFO - ==================================================\n2025-11-21 22:23:46,881 - INFO - 使用特征数量: 10\n2025-11-21 22:26:07,979 - INFO - 最佳超参数: {'max_depth': 5, 'n_estimators': 200}\n2025-11-21 22:26:08,064 - INFO - 训练集 - MSE: 0.2083, RMSE: 0.4564, MAE: 0.3310, R²: 0.8179\n2025-11-21 22:26:08,065 - INFO - 测试集 - MSE: 0.3313, RMSE: 0.5756, MAE: 0.4071, R²: 0.7113\n2025-11-21 22:26:08,066 - INFO - 交叉验证最佳得分 (R²): 0.7090\n2025-11-21 22:26:08,069 - INFO - \n================================================================================\n2025-11-21 22:26:08,070 - INFO - 所有模型性能对比（按测试集R²排序）:\n2025-11-21 22:26:08,070 - INFO - ================================================================================\n2025-11-21 22:26:08,075 - INFO -       model_name  test_r2  test_rmse  cv_best_score  feature_count\nGradientBoosting   0.7113     0.5756         0.7090             10\n    RandomForest   0.6991     0.5877         0.7005             10\n    DecisionTree   0.6091     0.6698         0.6114             10\n           Ridge   0.5869     0.6886         0.5978             10\nLinearRegression   0.5869     0.6886         0.5978             10\n           Lasso   0.5860     0.6893         0.5969             10\n2025-11-21 22:26:08,076 - INFO - \n================================================================================\n2025-11-21 22:26:08,077 - INFO - 最优模型: GradientBoosting\n2025-11-21 22:26:08,077 - INFO - ================================================================================\n2025-11-21 22:26:08,078 - INFO - 最优模型超参数: {'max_depth': 5, 'n_estimators': 200}\n2025-11-21 22:26:08,079 - INFO - 测试集 R²: 0.7113\n2025-11-21 22:26:08,079 - INFO - 测试集 RMSE: 0.5756\n2025-11-21 22:26:08,080 - INFO - 交叉验证最佳得分: 0.7090\n2025-11-21 22:26:08,081 - INFO - 训练集 R²: 0.8179\n2025-11-21 22:26:08,081 - INFO - 使用特征数量: 10\n2025-11-21 22:26:08,101 - INFO - \n模型保存完成:\n2025-11-21 22:26:08,102 - INFO - 最佳模型路径: ./saved_model/best_model_GradientBoostingRegressor.pkl\n2025-11-21 22:26:08,102 - INFO - 特征工程组件路径: ./saved_model/feature_engineering_objects.pkl\n2025-11-21 22:26:08,103 - INFO - 性能报告路径: ./saved_model/model_performance_report.csv\n2025-11-21 22:26:08,103 - INFO - \n================================================================================\n2025-11-21 22:26:08,104 - INFO - 模型训练流水线完成！\n2025-11-21 22:26:08,104 - INFO - ================================================================================\n",
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": "\n================================================================================\n训练结果摘要\n================================================================================\n最优模型: GradientBoosting\n最优模型路径: ./saved_model/best_model_GradientBoostingRegressor.pkl\n\n性能指标:\n  测试集 R² 得分: 0.7113\n  测试集 RMSE: 0.5756\n  测试集 MSE: 0.3313\n  测试集 MAE: 0.4071\n  交叉验证最佳得分: 0.7090\n\n特征工程信息:\n  选择的特征数量: 10\n  选择的特征: Index(['MEDINC', 'HOUSE_AGE', 'AVE_ROOMS', 'AVE_BEDRMS', 'AVE_OCCUP',\n       'LATITUDE', 'inter_0', 'inter_1', 'inter_3', 'inter_4'],\n      dtype='object')\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "\n# 运行训练流水线\ntry:\n    # 预测示例 - 使用用户提供的真实数据样本\n    print(\"\\n\" + \"=\"*80)\n    print(\"预测示例（基于真实数据样本）\")\n    print(\"=\"*80)\n\n    # 基于用户提供的数据创建预测样本（去掉DATA_SN和MED_HOUSE_VAL，仅保留特征列）\n    X_new = pd.DataFrame({\n        \"MEDINC\": [1.7348, 1.4113, 0.8172, 1.2171],\n        \"HOUSE_AGE\": [43, 52, 52, 52],\n        \"AVE_ROOMS\": [3.980237154, 4.295454545, 6.102459016, 4.5625],\n        \"AVE_BEDRMS\": [1.233201581, 1.104545455, 1.37295082, 1.121710526],\n        \"AVE_OCCUP\": [2.205533597, 2.618181818, 2.983606557, 3.532894737],\n        \"LATITUDE\": [37.82, 37.82, 37.82, 37.82],\n        \"LONGITUDE\": [-122.27, -122.28, -122.28, -122.28],\n        \"POPULATION\": [558, 576, 728, 1074]\n    })\n\n    # 显示输入特征\n    print(\"输入预测样本特征:\")\n    print(X_new.round(4).to_string(index=False))\n    print()\n\n    # 执行预测\n    predictions = predict_with_best_model(\"./saved_model\", X_new)\n\n    # 显示预测结果（同时显示真实值进行对比）\n    print(\"预测结果对比:\")\n    print(\"-\" * 80)\n    print(f\"{'样本序号':<8} {'真实房屋价值':<15} {'预测房屋价值':<15} {'误差':<10}\")\n    print(\"-\" * 80)\n\n    # 真实值（来自用户提供的数据）\n    true_values = [1.375, 0.831, 0.853, 0.803]\n\n    for i, (pred, true) in enumerate(zip(predictions, true_values)):\n        error = abs(pred - true)\n        print(f\"{i+1:<8} ${true:>13,.2f} ${pred:>13,.2f} {error:>10.4f}\")\n\n    # 计算平均误差\n    avg_error = np.mean([abs(pred - true) for pred, true in zip(predictions, true_values)])\n    print(\"-\" * 80)\n    print(f\"{'平均误差':<8} {'':<15} {'':<15} {avg_error:>10.4f}\")\n    print(f\"{'平均误差率':<8} {'':<15} {'':<15} {avg_error/np.mean(true_values)*100:>8.2f}%\")\n\nexcept Exception as e:\n    print(f\"程序执行失败: {str(e)}\")",
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": "2025-11-21 22:26:08,193 - INFO - 成功加载模型: ./saved_model/best_model_GradientBoostingRegressor.pkl\n2025-11-21 22:26:08,194 - INFO - 模型期望特征数量: 10\n2025-11-21 22:26:08,195 - INFO - 训练时选择的特征: Index(['MEDINC', 'HOUSE_AGE', 'AVE_ROOMS', 'AVE_BEDRMS', 'AVE_OCCUP',\n       'LATITUDE', 'inter_0', 'inter_1', 'inter_3', 'inter_4'],\n      dtype='object')\n2025-11-21 22:26:08,195 - INFO - 特征数量: 10\n2025-11-21 22:26:08,201 - INFO - 生成交互特征数量: 6\n2025-11-21 22:26:08,202 - INFO - 特征工程后总特征数量: 14\n2025-11-21 22:26:08,203 - INFO - 筛选后特征数量: 10\n2025-11-21 22:26:08,205 - INFO - 预测完成，生成 4 个预测结果\n",
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": "\n================================================================================\n预测示例（基于真实数据样本）\n================================================================================\n输入预测样本特征:\n MEDINC  HOUSE_AGE  AVE_ROOMS  AVE_BEDRMS  AVE_OCCUP  LATITUDE  LONGITUDE  POPULATION\n 1.7348         43     3.9802      1.2332     2.2055     37.82    -122.27         558\n 1.4113         52     4.2955      1.1045     2.6182     37.82    -122.28         576\n 0.8172         52     6.1025      1.3730     2.9836     37.82    -122.28         728\n 1.2171         52     4.5625      1.1217     3.5329     37.82    -122.28        1074\n\n预测结果对比:\n--------------------------------------------------------------------------------\n样本序号     真实房屋价值          预测房屋价值          误差        \n--------------------------------------------------------------------------------\n1        $         1.38 $         1.67     0.2934\n2        $         0.83 $         0.96     0.1260\n3        $         0.85 $         1.07     0.2195\n4        $         0.80 $         0.72     0.0835\n--------------------------------------------------------------------------------\n平均误差                                         0.1806\n平均误差率                                       18.70%\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "with open('./saved_model/linear.txt', 'w', encoding='utf-8') as f:\n    f.write(\"这是过程文件\")",
      "execution_count": 30,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.8.8",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}